{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as Data\n",
    "torch.manual_seed(8) # for reproduce\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import gc\n",
    "import sys\n",
    "sys.setrecursionlimit(50000)\n",
    "import pickle\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.set_default_tensor_type('torch.cuda.FloatTensor')\n",
    "# from tensorboardX import SummaryWriter\n",
    "torch.nn.Module.dump_patches = True\n",
    "import copy\n",
    "import pandas as pd\n",
    "#then import my own modules\n",
    "from AttentiveFP import Fingerprint, Fingerprint_viz, save_smiles_dicts, get_smiles_dicts, get_smiles_array, moltosvg_highlight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import f1_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from rdkit.Chem import rdMolDescriptors, MolSurf\n",
    "# from rdkit.Chem.Draw import SimilarityMaps\n",
    "from rdkit import Chem\n",
    "# from rdkit.Chem import AllChem\n",
    "from rdkit.Chem import QED\n",
    "%matplotlib inline\n",
    "from numpy.polynomial.polynomial import polyfit\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib\n",
    "from IPython.display import SVG, display\n",
    "import seaborn as sns; sns.set(color_codes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of all smiles:  1513\n",
      "number of successfully processed smiles:  1513\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAC/CAYAAAB+KF5fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFdhJREFUeJzt3X9M1Pf9B/DnHQfILw/mrtiJ1dbjmFaFNlKwEklVZHNVgrOzdWuD1aBZqawVI5mlpY1JDSLUQimCZVsz09iaFYI0WhkLzQplREtnh3KnrIbI+KH8ErDHj/t8/zDclxPu15uDu8PnIyEt78/rPvd+5SPP+9zn87nPySRJkkBERHaRO3sCRETuiOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkgOFJRCSA4UlEJIDhSUQkQOHsCThCd/cADAbXvDnUvHn+uH2739nTmFazvUf2597G9yeXyxAU5OeQ9c6K8DQYJJcNTwAuPTdHme09sj/3Nh398W07EZEAhicRkQCGJxGRAIYnEZGAWXHC6EE3YgD0wyNml3t7KqDgyySRQzE8ZwH98Ajqr7SbXR65NBgKb25qIkfi/ggRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAXr/i4qxdwwkAs/yeDkQuieHp4qxdwwkA4RrVDM2GiMbwbTsRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQCGJxGRAIYnEZEAhicRkQCGJxGRAKufba+trUVZWRm+/fZbtLW1QalUYuXKlXj11VcRFhZmUvv111/j+PHjuHr1Kvz8/BAXF4e0tDTMnTvXpG5gYAC5ubk4d+4c+vr6oFar8corr2D9+vWO7Y6IaJpY3fP85JNP0NraiqSkJBQXFyM9PR2tra3Ytm0bGhoajHV1dXVITk7G/PnzUVhYiIMHD6KqqgrJyckwGAwm60xJSUF5eTlSU1Nx4sQJqNVqpKSkoLq62vEdEhFNA6t7nm+99RbmzZtnMhYTE4P169fjo48+Ql5eHgDg6NGjCA0NxXvvvQe5/F4mq1QqvPzyyzh37hw2bdoEAKiurkZNTQ3y8/MRFxcHAIiOjkZLSwuOHDmC2NhYhzZIRDQdrO553h+cADB37lwsWrQIbW1tAID29nZcvnwZCQkJxuAEgDVr1iA4OBjnz583jl24cAEBAQEmb9FlMhkSExPR3NyMa9euTakhIqKZIHTCqKurCzqdDqGhoQAArVYLAMbfx9NoNNDpdMbfdTod1Gq1ScgCMB4/HVsXEZErs/tmyJIkISMjAwaDAbt27QIA9PT0AACUSuWEeqVSicbGRuPvPT09WLx48aR149dlj3nz/O1+zExSqQKEHyt1DSLAf47FGk9PhcUaX19vqH7iKzwHW0ylR3fA/tzbdPRnd3hmZWWhsrIS7777LpYsWWKyTCaTTfqY+8fN1VlbZs7t2/0wuOh3UahUAejsvCP8+EH9CO70/2ixZnjYcs3goB6do6PCc7Bmqj26Ovbn3sb3J5fLHLazZdfb9tzcXJSUlODQoUPYunWrcTwwMBDA5HuNvb29JnukgYGBZuuAyfdeiYhcjc3hefz4cRQWFuLAgQN46aWXTJaNHescf2xzjFarNTkWqlarcf369QmXL40d69RoNLbPnojISWwKz/z8fBQUFCA1NRW7d++esHz+/PlYvnw5ysvLTUKxtrYW7e3t2Lhxo3EsLi4OfX19qKqqMllHaWkpHn30UajVatFeiIhmjNVjniUlJcjLy8MzzzyDp59+2uTCeC8vLyxbtgwAkJaWhl27duH111/H9u3b0d7ejuzsbISHh+MXv/iF8TGxsbGIiorCoUOH0NPTg5CQEJSWluLixYsoKCiYhhaJiBzPanj+4x//MP537P/HLFiwwLgHuXr1ahQWFiIvLw/Jycnw8/PDhg0bcODAAXh4eBgfI5PJUFBQgJycHOTm5ho/npmfn49169Y5sjciomkjkyTJNU9T22E2n20f0Nv2ve3faTvNLo9cGgw/b7svrLDZg3S2djZ6kPpz2tl2IiK6Z/p2R2jWGTEA+uGRCeNS1yAG9ffGvT0VUPAlmR4ADE+ymX548kMIAf5zjBfpRy4NhmIaDxEQuQruIxARCWB4EhEJ4PsrAmD+eOZ4LnpBA5FTMDwJgPnjmeOFa1QzNBsi18fwfADI5DIM6LlXSeRIDM8HgH541OJF9AD3KonsxRNGREQCGJ5ERAIYnkREAhieREQCGJ5ERAIYnkREAhieREQCGJ5ERAIYnkREAhieREQCGJ5ERAIYnkREAhieREQCGJ5ERAJ4SzpyKGv3DuW3a9JswfAkh7J271B+uybNFtwHICISwPAkIhLA8CQiEsDwJCISwPAkIhLA8CQiEsDwJCISwPAkIhLA8CQiEsDwJCISwPAkIhLADxk70YgB0A+bv4kGABikGZoMEdmF4elE+uER1F9pt1gTrlHN0GyIyB58205EJIDhSUQkgOFJRCSA4UlEJIDhSUQkwKbwbGtrw+HDh/HCCy/giSeeQFhYGOrq6iatLS8vx5YtW7BixQqsXbsW2dnZ0Ov1E+pu3bqFgwcPIioqChEREdixYwcuXbo0tW6IiGaITeF548YNVFRUwNfXF9HR0WbrysrKkJaWhieffBLFxcXYs2cPTp06hfT0dJM6vV6PpKQk1NfXIyMjA/n5+fDz80NSUhIaGxun1hER0Qyw6TrPyMhI1NbWAgAqKytRVVU1oWZ0dBRHjx7FunXrkJmZCQCIjo6Gp6cnMjIykJSUhPDwcADAmTNnoNPp8Le//Q2PP/44AOCpp57CL3/5S+Tk5ODkyZOO6I2IaNrYtOcpl1sva2hoQGdnJxITE03GN2/eDE9PT5w/f944VllZCY1GYwxOAPDy8sKzzz6Lmpoa9Pf32zp/IiKncNgJI51OBwAIDQ01Gffx8cHChQuNy8dqNRrNhHWEhYVhdHQUzc3NjpoWEdG0cFh49vT0AACUSuWEZUql0rh8rNZcHQB0d3c7alpERNPC4Z9tl8lkNo2bq7O2bDLz5vnbVT/TVKqAScelrkEE+M+x+FhPT8WUaxyxDms1Y+PW1uPr6w3VT3wtPo8rMrcNZwv2Zz+HhWdgYCCAe3uVQUFBJst6e3sREhJiUjt+T3R83fh12er27X4YXPT2QypVADo770y6bFA/gjv9P1p8/PDw1GscsQ5LNQH+c4zj1tYzOKhH5+ioxedxNZa24WzwIPUnl8sctrPlsLftarUaAEyObQLA3bt30dLSYnIsVK1WQ6vVTlhHU1MTPDw88NhjjzlqWkRE08Jh4RkREQGVSoWysjKT8bNnz2J4eBgbN240jsXFxUGr1eLKlSvGsaGhIVRUVGD16tXw93ftt+FERDa/bT937hwA4PLlywCA+vp6dHd3w8fHB7GxsVAoFNi/fz/S09PxzjvvID4+HtevX0d2djbi4+MRERFhXNe2bdtw6tQppKSkYP/+/VAqlfj444/R0dGB9957z8EtEhE5ns3hmZqaavJ7Xl4eAGDBggXGi+YTExMhl8tx8uRJfPrppwgKCsLzzz+Pffv2mTzW29sbf/nLX5CVlYXMzEzo9XosW7YMJSUlWL58+VR7IiKadjaHZ1NTk011CQkJSEhIsFqnUqlw9OhRW5+eiMil8K5KREQCGJ5ERAIYnkREAhieREQCGJ5ERAL4ve00o2RyGQb0IxZrvD0VUPBlnVwcw5NmlH54FN9pOy3WRC4NhsKb/zTJtfH1nYhIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAMOTiEgAw5OISADDk4hIAL8ohlwOvySO3AHDk1wOvySO3AFfu4mIBDA8iYgEMDyJiATwoNE0GjEAHV2DGDRz8sMgzfCEiMhhGJ7TSD88gqvNt3Gn/8dJl4drVDM8IyJyFL5tJyISwPAkIhLA8CQiEsDwJCISwPAkIhLAs+3klqx9/p2ffafpxvAUNGK4dymSJbyOc/pY+/w7P/tO043/ugTph0dQf6XdYg2v4ySavfjGhohIAMOTiEgAw5OISIDTwnNgYACHDx9GTEwMVq5cia1bt+Lvf/+7s6ZDRGQXp50wSklJQWNjI9LS0hASEoLPP/8cKSkpKCwsRGxs7LQ9ry1nyT0VCgyP8Ew6EZnnlPCsrq5GTU0N8vPzERcXBwCIjo5GS0sLjhw5Mq3haetZcmtfA8Ez6a7NUd+DZO22grauh2Yfp4TnhQsXEBAQgPXr1xvHZDIZEhMTkZGRgWvXrkGtVjtjajRLOOp7kKzdVtDW9dDs45QtrtPpoFarIZebvlyHhYUBALRarV3hKZfLbK5VeMjhO8dzxmp8vBUYHZm8bqbmMt3PM75HZ8/FrnV4ekA/YrBYI5db3oa2rMdL4QEPK3umowZgaGTUYo0t6xFlz9+QOxrrz5F9yiRJmvGjd/Hx8Vi8eDFOnDhhMv7DDz8gPj4eb731Fnbs2DHT0yIispnTjtTIZOZfASwtIyJyBU4Jz8DAQPT09EwY7+3tBQAolcqZnhIRkV2cEp5qtRrXr1+HwWB6nEir1QIANBqNM6ZFRGQzp4RnXFwc+vr6UFVVZTJeWlqKRx99lGfaicjlOeVse2xsLKKionDo0CH09PQgJCQEpaWluHjxIgoKCpwxJSIiuzjlbDsA9Pf3IycnB+fPn0dfXx/UajVeeeUVbNiwwRnTISKyi9PCk4jInfFDZUREAhieREQCGJ5TUFtbi/T0dMTHxyM8PBxr165FSkoKmpqaJtR+/fXX+M1vfoOVK1di9erVePPNN9HX1+eEWU9NXl4ewsLCkJCQMGGZu/ZYV1eHl19+GatWrUJ4eDg2bdqE06dPm9SUl5djy5YtWLFiBdauXYvs7Gzo9Xonzdh2jY2N+P3vf4+YmBhERERg06ZNKCoqwtDQkEmdO2y7trY2HD58GC+88AKeeOIJhIWFoa6ubtJaW7fXrVu3cPDgQURFRSEiIgI7duzApUuXbJoPw3MKPvnkE7S2tiIpKQnFxcVIT09Ha2srtm3bhoaGBmNdXV0dkpOTMX/+fBQWFuLgwYOoqqpCcnLyhGtdXZlOp0NxcTF++tOfTljmrj1+/vnn2LlzJxYuXIicnBwUFhbit7/9LYaHh401ZWVlSEtLw5NPPoni4mLs2bMHp06dQnp6uhNnbt3169fx/PPP4+bNm/jjH/+IDz/8EHFxccjNzcUbb7xhrHOXbXfjxg1UVFTA19cX0dHRZuts3V56vR5JSUmor69HRkYG8vPz4efnh6SkJDQ2NlqfkETCbt26NWGst7dXWrVqlZSSkmIc+/Wvfy0lJCRIo6OjxrF//vOfkkajkSoqKmZkrlM1OjoqPffcc9I777wj/e53v5O2bNlistwde2xtbZVWrlwpFRUVma0ZGRmR1qxZI+3du9dk/PTp05JGo5EaGhqme5rC3n//fUmj0Ug3btwwGU9LS5OWLVsmDQ0NSZLkPttu/PwuXLggaTQa6ZtvvjGpsWd7/fWvf5U0Go30/fffG8f0er20bt06adeuXVbnwz3PKZg3b96Esblz52LRokVoa2sDALS3t+Py5ctISEgwuYvUmjVrEBwcjPPnz8/YfKfiz3/+M9ra2vDaa69NWOauPZ45cwYA8OKLL5qtaWhoQGdnJxITE03GN2/eDE9PT5ftDQAUinuXcfv7+5uMBwQEQKFQwMPDw6223f13YZuMPdursrISGo0Gjz/+uHHMy8sLzz77LGpqatDf3295PnbOn6zo6uqCTqdDaGgogP//yOnY7+NpNBrodLoZnZ+IlpYWvP/++3jzzTcn/CEC7ttjfX09lixZgi+//BLx8fFYunSp8fjY2DHBsbnf35uPjw8WLlzosr0BQEJCAgIDA5GZmYmWlhb09/ejsrLSeKhCLpe77bYzx57tpdPpJv0oeFhYGEZHR9Hc3GzxuXgHVweSJAkZGRkwGAzYtWsXABhvgDLZzU6USqVtx1acSJIkvPHGG4iJiTH7AQZ37bGjowMdHR04fPgwUlNToVar8c0336CoqAj/+9//cOzYMau9TXaDG1fxs5/9DKdPn57w4ZO9e/fiD3/4AwD33Xbm2LO9enp6zNYBQHd3t8XnYng6UFZWFiorK/Huu+9iyZIlJsvM3WbP1W+/9+mnn+L777/HF198YbXW3XqUJAkDAwPIycnBr371KwBAVFQUfvzxR5SUlGDfvn3GWnfrDQBu3ryJvXv3QqVS4YMPPkBAQADq6+tx4sQJyGQyY4AC7tmfJbb2M5VbYzI8HSQ3NxclJSU4dOgQtm7dahwPDAwEALO34HPl2+91dXXh6NGj2LNnD3x8fIyXroyMjMBgMKCvrw/e3t5u2+PYvGNiYkzG165di5KSEvznP/8x6S0oKMikrre3FyEhITMzWQHHjh3DwMAASktLMWfOHAD3XhwA4IMPPsC2bdvcdtuZY8/2snZrzLF1mcNjng5w/PhxFBYW4sCBA3jppZdMlo0de5ns2JFWq530WJOraG9vx507d3Ds2DFERkYafy5dugStVovIyEjk5eW5bY/Wbn0ol8uNd/i6v7e7d++ipaXFZXsD7l3jqVarjcE5Zvny5TAYDGhubnbbbWeOPdtLrVYbj/mO19TUBA8PDzz22GMWn4vhOUX5+fkoKChAamoqdu/ePWH5/PnzsXz5cpSXl5tcM1dbW4v29nZs3LhxJqdrl0ceeQQff/zxhJ+f//znxmXbt2932x7Hvrm1urraZLy6uhoymQwrVqxAREQEVCoVysrKTGrOnj2L4eFhl+0NAB566CHodDrcvXvXZPzbb78FAAQHB7vttjPHnu0VFxcHrVaLK1euGMeGhoZQUVGB1atXT3pydDyPzMzMTIfO/gFSUlKCnJwcPPPMM0hMTERbW5vxp6urCyrVva8nfuSRR1BSUoJr165BqVTi4sWLePvttxEaGor09HSbLsFwBi8vL4SEhEz4+eKLL2AwGJCWlmZ8W+eOPS5atAj//ve/cfr0aXh7e2NwcBCfffYZ/vSnP2H79u3YsmUL5HI5goKCUFRUhO7ubsyZMwdfffUVsrKysG7dOuzcudPZbZgVEBCAzz77DPX19fD390dHRwfOnDmDjz76CNHR0caTmu607c6dO4dr167hu+++w6VLlxASEoKuri7cvHkTixcvtmt7hYWF4csvv0R5eTlUKhU6Ojpw5MgRNDU1ITs7Gw899JDFufCuSlPw4osv4l//+tekyxYsWGBys+evvvoKeXl5uHr1Kvz8/LBhwwYcOHDA7Y4pAff67uvrm/Dq7o49Dg4OIi8vD2fPnkV3dzcefvhhPPfcc9i9e7dJaJSVleHkyZP473//i6CgIGzevBn79u2b8JbY1dTU1KCoqAharRaDg4NYsGABNm3ahJ07d8LX19dY5y7bbuwbdu93/9+brdurs7MTWVlZqK6uhl6vx7Jly7B//36sWrXK6lwYnkREAlxnf5yIyI0wPImIBDA8iYgEMDyJiAQwPImIBDA8iYgEMDyJiAQwPImIBDA8iYgE/B8ERM9pqKBwIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "task_name = 'BACE'\n",
    "tasks = ['Class']\n",
    "raw_filename = \"../data/bace.csv\"\n",
    "feature_filename = raw_filename.replace('.csv','.pickle')\n",
    "filename = raw_filename.replace('.csv','')\n",
    "prefix_filename = raw_filename.split('/')[-1].replace('.csv','')\n",
    "smiles_tasks_df = pd.read_csv(raw_filename)\n",
    "smilesList = smiles_tasks_df.mol.values\n",
    "print(\"number of all smiles: \",len(smilesList))\n",
    "atom_num_dist = []\n",
    "remained_smiles = []\n",
    "canonical_smiles_list = []\n",
    "for smiles in smilesList:\n",
    "    try:        \n",
    "        mol = Chem.MolFromSmiles(smiles)\n",
    "        atom_num_dist.append(len(mol.GetAtoms()))\n",
    "        remained_smiles.append(smiles)\n",
    "        canonical_smiles_list.append(Chem.MolToSmiles(Chem.MolFromSmiles(smiles), isomericSmiles=True))\n",
    "    except:\n",
    "        print(\"not successfully processed smiles: \", smiles)\n",
    "        pass\n",
    "print(\"number of successfully processed smiles: \", len(remained_smiles))\n",
    "smiles_tasks_df = smiles_tasks_df[smiles_tasks_df[\"mol\"].isin(remained_smiles)]\n",
    "# print(smiles_tasks_df)\n",
    "smiles_tasks_df['cano_smiles'] =canonical_smiles_list\n",
    "assert canonical_smiles_list[8]==Chem.MolToSmiles(Chem.MolFromSmiles(smiles_tasks_df['cano_smiles'][8]), isomericSmiles=True)\n",
    "\n",
    "plt.figure(figsize=(5, 3))\n",
    "sns.set(font_scale=1.5)\n",
    "ax = sns.distplot(atom_num_dist, bins=28, kde=False)\n",
    "plt.tight_layout()\n",
    "# plt.savefig(\"atom_num_dist_\"+prefix_filename+\".png\",dpi=200)\n",
    "plt.show()\n",
    "plt.close()\n",
    "\n",
    "# print(len([i for i in atom_num_dist if i<51]),len([i for i in atom_num_dist if i>50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed = 88\n",
    "start_time = str(time.ctime()).replace(':','-').replace(' ','_')\n",
    "start = time.time()\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 800\n",
    "p_dropout = 0.1\n",
    "fingerprint_dim = 150\n",
    "\n",
    "radius = 3\n",
    "T = 2\n",
    "weight_decay = 2.9 # also known as l2_regularization_lambda\n",
    "learning_rate = 3.5\n",
    "per_task_output_units_num = 2 # for classification model with 2 classes\n",
    "output_units_num = len(tasks) * per_task_output_units_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mol</th>\n",
       "      <th>CID</th>\n",
       "      <th>Class</th>\n",
       "      <th>Model</th>\n",
       "      <th>pIC50</th>\n",
       "      <th>MW</th>\n",
       "      <th>AlogP</th>\n",
       "      <th>HBA</th>\n",
       "      <th>HBD</th>\n",
       "      <th>RB</th>\n",
       "      <th>...</th>\n",
       "      <th>PEOE7 (PEOE7)</th>\n",
       "      <th>PEOE8 (PEOE8)</th>\n",
       "      <th>PEOE9 (PEOE9)</th>\n",
       "      <th>PEOE10 (PEOE10)</th>\n",
       "      <th>PEOE11 (PEOE11)</th>\n",
       "      <th>PEOE12 (PEOE12)</th>\n",
       "      <th>PEOE13 (PEOE13)</th>\n",
       "      <th>PEOE14 (PEOE14)</th>\n",
       "      <th>canvasUID</th>\n",
       "      <th>cano_smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 596 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [mol, CID, Class, Model, pIC50, MW, AlogP, HBA, HBD, RB, HeavyAtomCount, ChiralCenterCount, ChiralCenterCountAllPossible, RingCount, PSA, Estate, MR, Polar, sLi_Key, ssBe_Key, ssssBem_Key, sBH2_Key, ssBH_Key, sssB_Key, ssssBm_Key, sCH3_Key, dCH2_Key, ssCH2_Key, tCH_Key, dsCH_Key, aaCH_Key, sssCH_Key, ddC_Key, tsC_Key, dssC_Key, aasC_Key, aaaC_Key, ssssC_Key, sNH3_Key, sNH2_Key, ssNH2_Key, dNH_Key, ssNH_Key, aaNH_Key, tN_Key, sssNH_Key, dsN_Key, aaN_Key, sssN_Key, ddsN_Key, aasN_Key, ssssN_Key, daaN_Key, sOH_Key, dO_Key, ssO_Key, aaO_Key, aOm_Key, sOm_Key, sF_Key, sSiH3_Key, ssSiH2_Key, sssSiH_Key, ssssSi_Key, sPH2_Key, ssPH_Key, sssP_Key, dsssP_Key, ddsP_Key, sssssP_Key, sSH_Key, dS_Key, ssS_Key, aaS_Key, dssS_Key, ddssS_Key, ssssssS_Key, Sm_Key, sCl_Key, sGeH3_Key, ssGeH2_Key, sssGeH_Key, ssssGe_Key, sAsH2_Key, ssAsH_Key, sssAs_Key, dsssAs_Key, ddsAs_Key, sssssAs_Key, sSeH_Key, dSe_Key, ssSe_Key, aaSe_Key, dssSe_Key, ssssssSe_Key, ddssSe_Key, sBr_Key, sSnH3_Key, ssSnH2_Key, sssSnH_Key, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 596 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# smilesList = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())<151]\n",
    "# uncovered = [smiles for smiles in canonical_smiles_list if len(Chem.MolFromSmiles(smiles).GetAtoms())>150]\n",
    "\n",
    "# smiles_tasks_df = smiles_tasks_df[~smiles_tasks_df[\"cano_smiles\"].isin(uncovered)]\n",
    "\n",
    "if os.path.isfile(feature_filename):\n",
    "    feature_dicts = pickle.load(open(feature_filename, \"rb\" ))\n",
    "else:\n",
    "    feature_dicts = save_smiles_dicts(smilesList,filename)\n",
    "# feature_dicts = get_smiles_dicts(smilesList)\n",
    "\n",
    "remained_df = smiles_tasks_df[smiles_tasks_df[\"cano_smiles\"].isin(feature_dicts['smiles_to_atom_mask'].keys())]\n",
    "uncovered_df = smiles_tasks_df.drop(remained_df.index)\n",
    "uncovered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = []\n",
    "for i,task in enumerate(tasks):    \n",
    "    negative_df = remained_df[remained_df[task] == 0][[\"mol\",task]]\n",
    "    positive_df = remained_df[remained_df[task] == 1][[\"mol\",task]]\n",
    "    weights.append([(positive_df.shape[0]+negative_df.shape[0])/negative_df.shape[0],\\\n",
    "                    (positive_df.shape[0]+negative_df.shape[0])/positive_df.shape[0]])\n",
    "\n",
    "test_df = remained_df.sample(frac=1/10, random_state=random_seed) # test set\n",
    "training_data = remained_df.drop(test_df.index) # training data\n",
    "\n",
    "# training data is further divided into validation set and train set\n",
    "valid_df = training_data.sample(frac=1/9, random_state=random_seed) # validation set\n",
    "train_df = training_data.drop(valid_df.index) # train set\n",
    "train_df = train_df.reset_index(drop=True)\n",
    "valid_df = valid_df.reset_index(drop=True)\n",
    "test_df = test_df.reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "649206\n",
      "atom_fc.weight torch.Size([150, 39])\n",
      "atom_fc.bias torch.Size([150])\n",
      "neighbor_fc.weight torch.Size([150, 49])\n",
      "neighbor_fc.bias torch.Size([150])\n",
      "GRUCell.0.weight_ih torch.Size([450, 150])\n",
      "GRUCell.0.weight_hh torch.Size([450, 150])\n",
      "GRUCell.0.bias_ih torch.Size([450])\n",
      "GRUCell.0.bias_hh torch.Size([450])\n",
      "GRUCell.1.weight_ih torch.Size([450, 150])\n",
      "GRUCell.1.weight_hh torch.Size([450, 150])\n",
      "GRUCell.1.bias_ih torch.Size([450])\n",
      "GRUCell.1.bias_hh torch.Size([450])\n",
      "GRUCell.2.weight_ih torch.Size([450, 150])\n",
      "GRUCell.2.weight_hh torch.Size([450, 150])\n",
      "GRUCell.2.bias_ih torch.Size([450])\n",
      "GRUCell.2.bias_hh torch.Size([450])\n",
      "align.0.weight torch.Size([1, 300])\n",
      "align.0.bias torch.Size([1])\n",
      "align.1.weight torch.Size([1, 300])\n",
      "align.1.bias torch.Size([1])\n",
      "align.2.weight torch.Size([1, 300])\n",
      "align.2.bias torch.Size([1])\n",
      "attend.0.weight torch.Size([150, 150])\n",
      "attend.0.bias torch.Size([150])\n",
      "attend.1.weight torch.Size([150, 150])\n",
      "attend.1.bias torch.Size([150])\n",
      "attend.2.weight torch.Size([150, 150])\n",
      "attend.2.bias torch.Size([150])\n",
      "mol_GRUCell.weight_ih torch.Size([450, 150])\n",
      "mol_GRUCell.weight_hh torch.Size([450, 150])\n",
      "mol_GRUCell.bias_ih torch.Size([450])\n",
      "mol_GRUCell.bias_hh torch.Size([450])\n",
      "mol_align.weight torch.Size([1, 300])\n",
      "mol_align.bias torch.Size([1])\n",
      "mol_attend.weight torch.Size([150, 150])\n",
      "mol_attend.bias torch.Size([150])\n",
      "output.weight torch.Size([2, 150])\n",
      "output.bias torch.Size([2])\n"
     ]
    }
   ],
   "source": [
    "x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array([canonical_smiles_list[0]],feature_dicts)\n",
    "num_atom_features = x_atom.shape[-1]\n",
    "num_bond_features = x_bonds.shape[-1]\n",
    "\n",
    "loss_function = [nn.CrossEntropyLoss(torch.Tensor(weight),reduction='mean') for weight in weights]\n",
    "model = Fingerprint(radius, T, num_atom_features,num_bond_features,\n",
    "            fingerprint_dim, output_units_num, p_dropout)\n",
    "model.cuda()\n",
    "# tensorboard = SummaryWriter(log_dir=\"runs/\"+start_time+\"_\"+prefix_filename+\"_\"+str(fingerprint_dim)+\"_\"+str(p_dropout))\n",
    "\n",
    "# optimizer = optim.Adam(model.parameters(), learning_rate, weight_decay=weight_decay)\n",
    "optimizer = optim.Adam(model.parameters(), 10**-learning_rate, weight_decay=10**-weight_decay)\n",
    "model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "params = sum([np.prod(p.size()) for p in model_parameters])\n",
    "print(params)\n",
    "for name, param in model.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, param.data.shape)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, dataset, optimizer, loss_function):\n",
    "    model.train()\n",
    "    np.random.seed(epoch)\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    #shuffle them\n",
    "    np.random.shuffle(valList)\n",
    "    batch_list = []\n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, train_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[train_batch,:]\n",
    "        smiles_list = batch_df.cano_smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "#         print(torch.Tensor(x_atom).size(),torch.Tensor(x_bonds).size(),torch.cuda.LongTensor(x_atom_index).size(),torch.cuda.LongTensor(x_bond_index).size(),torch.Tensor(x_mask).size())\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss = 0.0\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where(y_val != -1)[0]\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "\n",
    "            loss += loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "        # Step 5. Do the backward pass and update the gradient\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "def eval(model, dataset):\n",
    "    model.eval()\n",
    "    y_val_list = {}\n",
    "    y_pred_list = {}\n",
    "    losses_list = []\n",
    "    valList = np.arange(0,dataset.shape[0])\n",
    "    batch_list = []\n",
    "    for i in range(len(tasks)):\n",
    "        y_val_list[i] = []\n",
    "        y_pred_list[i] = []\n",
    "    \n",
    "    for i in range(0, dataset.shape[0], batch_size):\n",
    "        batch = valList[i:i+batch_size]\n",
    "        batch_list.append(batch)   \n",
    "    for counter, eval_batch in enumerate(batch_list):\n",
    "        batch_df = dataset.loc[eval_batch,:]\n",
    "        smiles_list = batch_df.cano_smiles.values\n",
    "        \n",
    "        x_atom, x_bonds, x_atom_index, x_bond_index, x_mask, smiles_to_rdkit_list = get_smiles_array(smiles_list,feature_dicts)\n",
    "        atoms_prediction, mol_prediction = model(torch.Tensor(x_atom),torch.Tensor(x_bonds),torch.cuda.LongTensor(x_atom_index),torch.cuda.LongTensor(x_bond_index),torch.Tensor(x_mask))\n",
    "        atom_pred = atoms_prediction.data[:,:,1].unsqueeze(2).cpu().numpy()\n",
    "        for i,task in enumerate(tasks):\n",
    "            y_pred = mol_prediction[:, i * per_task_output_units_num:(i + 1) *\n",
    "                                    per_task_output_units_num]\n",
    "            y_val = batch_df[task].values\n",
    "\n",
    "            validInds = np.where((y_val==0) | (y_val==1))[0]\n",
    "#             validInds = np.where((y_val=='0') | (y_val=='1'))[0]\n",
    "#             print(validInds)\n",
    "            if len(validInds) == 0:\n",
    "                continue\n",
    "            y_val_adjust = np.array([y_val[v] for v in validInds]).astype(float)\n",
    "            validInds = torch.cuda.LongTensor(validInds).squeeze()\n",
    "            y_pred_adjust = torch.index_select(y_pred, 0, validInds)\n",
    "#             print(validInds)\n",
    "            loss = loss_function[i](\n",
    "                y_pred_adjust,\n",
    "                torch.cuda.LongTensor(y_val_adjust))\n",
    "#             print(y_pred_adjust)\n",
    "            y_pred_adjust = F.softmax(y_pred_adjust,dim=-1).data.cpu().numpy()[:,1]\n",
    "            losses_list.append(loss.cpu().detach().numpy())\n",
    "        \n",
    "            y_val_list[i].extend(y_val_adjust)\n",
    "            y_pred_list[i].extend(y_pred_adjust)\n",
    "                \n",
    "    eval_roc = [roc_auc_score(y_val_list[i], y_pred_list[i]) for i in range(len(tasks))]\n",
    "#     eval_prc = [auc(precision_recall_curve(y_val_list[i], y_pred_list[i])[1],precision_recall_curve(y_val_list[i], y_pred_list[i])[0]) for i in range(len(tasks))]\n",
    "#     eval_precision = [precision_score(y_val_list[i],\n",
    "#                                      (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "#     eval_recall = [recall_score(y_val_list[i],\n",
    "#                                (np.array(y_pred_list[i]) > 0.5).astype(int)) for i in range(len(tasks))]\n",
    "    eval_loss = np.array(losses_list).mean()\n",
    "    \n",
    "    return eval_roc, eval_loss #eval_prc, eval_precision, eval_recall, \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:\t0\n",
      "train_roc:[0.32058718224131755]\n",
      "valid_roc:[0.3630281690140846]\n",
      "\n",
      "EPOCH:\t1\n",
      "train_roc:[0.679916825029607]\n",
      "valid_roc:[0.6172535211267606]\n",
      "\n",
      "EPOCH:\t2\n",
      "train_roc:[0.6856977608857308]\n",
      "valid_roc:[0.6198943661971831]\n",
      "\n",
      "EPOCH:\t3\n",
      "train_roc:[0.6903632708143986]\n",
      "valid_roc:[0.6251760563380282]\n",
      "\n",
      "EPOCH:\t4\n",
      "train_roc:[0.6950921259191936]\n",
      "valid_roc:[0.6304577464788733]\n",
      "\n",
      "EPOCH:\t5\n",
      "train_roc:[0.6997245861907516]\n",
      "valid_roc:[0.6376760563380282]\n",
      "\n",
      "EPOCH:\t6\n",
      "train_roc:[0.7080833953014405]\n",
      "valid_roc:[0.6415492957746479]\n",
      "\n",
      "EPOCH:\t7\n",
      "train_roc:[0.713924922195599]\n",
      "valid_roc:[0.6436619718309858]\n",
      "\n",
      "EPOCH:\t8\n",
      "train_roc:[0.7238095238095238]\n",
      "valid_roc:[0.6545774647887325]\n",
      "\n",
      "EPOCH:\t9\n",
      "train_roc:[0.7345065961607316]\n",
      "valid_roc:[0.6632042253521127]\n",
      "\n",
      "EPOCH:\t10\n",
      "train_roc:[0.7450384202263902]\n",
      "valid_roc:[0.6693661971830986]\n",
      "\n",
      "EPOCH:\t11\n",
      "train_roc:[0.7431656063235011]\n",
      "valid_roc:[0.6711267605633803]\n",
      "\n",
      "EPOCH:\t12\n",
      "train_roc:[0.7468286099865048]\n",
      "valid_roc:[0.6748239436619718]\n",
      "\n",
      "EPOCH:\t13\n",
      "train_roc:[0.7450769781596849]\n",
      "valid_roc:[0.6720070422535211]\n",
      "\n",
      "EPOCH:\t14\n",
      "train_roc:[0.7514996281913575]\n",
      "valid_roc:[0.676056338028169]\n",
      "\n",
      "EPOCH:\t15\n",
      "train_roc:[0.7581412322013825]\n",
      "valid_roc:[0.6897887323943662]\n",
      "\n",
      "EPOCH:\t16\n",
      "train_roc:[0.7672532980803658]\n",
      "valid_roc:[0.7008802816901409]\n",
      "\n",
      "EPOCH:\t17\n",
      "train_roc:[0.770145143077474]\n",
      "valid_roc:[0.7049295774647888]\n",
      "\n",
      "EPOCH:\t18\n",
      "train_roc:[0.7724778980418079]\n",
      "valid_roc:[0.7044014084507043]\n",
      "\n",
      "EPOCH:\t19\n",
      "train_roc:[0.7756258778815169]\n",
      "valid_roc:[0.7088028169014085]\n",
      "\n",
      "EPOCH:\t20\n",
      "train_roc:[0.7786113635737696]\n",
      "valid_roc:[0.7216549295774648]\n",
      "\n",
      "EPOCH:\t21\n",
      "train_roc:[0.7816849816849818]\n",
      "valid_roc:[0.717781690140845]\n",
      "\n",
      "EPOCH:\t22\n",
      "train_roc:[0.7846016139249222]\n",
      "valid_roc:[0.7213028169014084]\n",
      "\n",
      "EPOCH:\t23\n",
      "train_roc:[0.7897766394006995]\n",
      "valid_roc:[0.7348591549295774]\n",
      "\n",
      "EPOCH:\t24\n",
      "train_roc:[0.792186510231623]\n",
      "valid_roc:[0.7371478873239437]\n",
      "\n",
      "EPOCH:\t25\n",
      "train_roc:[0.7858850422760197]\n",
      "valid_roc:[0.7160211267605633]\n",
      "\n",
      "EPOCH:\t26\n",
      "train_roc:[0.7913354815610454]\n",
      "valid_roc:[0.7257042253521127]\n",
      "\n",
      "EPOCH:\t27\n",
      "train_roc:[0.7967225756699441]\n",
      "valid_roc:[0.7383802816901408]\n",
      "\n",
      "EPOCH:\t28\n",
      "train_roc:[0.8019141259742764]\n",
      "valid_roc:[0.7463028169014083]\n",
      "\n",
      "EPOCH:\t29\n",
      "train_roc:[0.8021757690930624]\n",
      "valid_roc:[0.7455985915492958]\n",
      "\n",
      "EPOCH:\t30\n",
      "train_roc:[0.804478228538379]\n",
      "valid_roc:[0.748943661971831]\n",
      "\n",
      "EPOCH:\t31\n",
      "train_roc:[0.8077088325208626]\n",
      "valid_roc:[0.7540492957746479]\n",
      "\n",
      "EPOCH:\t32\n",
      "train_roc:[0.8110055358175658]\n",
      "valid_roc:[0.7584507042253521]\n",
      "\n",
      "EPOCH:\t33\n",
      "train_roc:[0.8146740477567546]\n",
      "valid_roc:[0.7612676056338028]\n",
      "\n",
      "EPOCH:\t34\n",
      "train_roc:[0.8169021454735741]\n",
      "valid_roc:[0.7639084507042254]\n",
      "\n",
      "EPOCH:\t35\n",
      "train_roc:[0.8209066622600458]\n",
      "valid_roc:[0.7718309859154929]\n",
      "\n",
      "EPOCH:\t36\n",
      "train_roc:[0.8243493348756507]\n",
      "valid_roc:[0.7737676056338029]\n",
      "\n",
      "EPOCH:\t37\n",
      "train_roc:[0.8277672202484232]\n",
      "valid_roc:[0.7839788732394366]\n",
      "\n",
      "EPOCH:\t38\n",
      "train_roc:[0.8308628714643753]\n",
      "valid_roc:[0.784330985915493]\n",
      "\n",
      "EPOCH:\t39\n",
      "train_roc:[0.833986064061252]\n",
      "valid_roc:[0.7896126760563381]\n",
      "\n",
      "EPOCH:\t40\n",
      "train_roc:[0.8368145638822332]\n",
      "valid_roc:[0.7955985915492958]\n",
      "\n",
      "EPOCH:\t41\n",
      "train_roc:[0.8388691508992261]\n",
      "valid_roc:[0.8008802816901409]\n",
      "\n",
      "EPOCH:\t42\n",
      "train_roc:[0.8408383596353521]\n",
      "valid_roc:[0.8029929577464789]\n",
      "\n",
      "EPOCH:\t43\n",
      "train_roc:[0.8438679115370845]\n",
      "valid_roc:[0.8059859154929577]\n",
      "\n",
      "EPOCH:\t44\n",
      "train_roc:[0.8450935029882397]\n",
      "valid_roc:[0.8080985915492959]\n",
      "\n",
      "EPOCH:\t45\n",
      "train_roc:[0.845806824754193]\n",
      "valid_roc:[0.8047535211267606]\n",
      "\n",
      "EPOCH:\t46\n",
      "train_roc:[0.8472031727670826]\n",
      "valid_roc:[0.8063380281690141]\n",
      "\n",
      "EPOCH:\t47\n",
      "train_roc:[0.8491586108127461]\n",
      "valid_roc:[0.8072183098591549]\n",
      "\n",
      "EPOCH:\t48\n",
      "train_roc:[0.8511112947203172]\n",
      "valid_roc:[0.8040492957746479]\n",
      "\n",
      "EPOCH:\t49\n",
      "train_roc:[0.8482717783469664]\n",
      "valid_roc:[0.8045774647887324]\n",
      "\n",
      "EPOCH:\t50\n",
      "train_roc:[0.8481753835137293]\n",
      "valid_roc:[0.8065140845070423]\n",
      "\n",
      "EPOCH:\t51\n",
      "train_roc:[0.8525131510093917]\n",
      "valid_roc:[0.8079225352112676]\n",
      "\n",
      "EPOCH:\t52\n",
      "train_roc:[0.8536588724558649]\n",
      "valid_roc:[0.8105633802816902]\n",
      "\n",
      "EPOCH:\t53\n",
      "train_roc:[0.8566250791814701]\n",
      "valid_roc:[0.8129401408450704]\n",
      "\n",
      "EPOCH:\t54\n",
      "train_roc:[0.8576964388994465]\n",
      "valid_roc:[0.816549295774648]\n",
      "\n",
      "EPOCH:\t55\n",
      "train_roc:[0.8603541821586934]\n",
      "valid_roc:[0.8193661971830986]\n",
      "\n",
      "EPOCH:\t56\n",
      "train_roc:[0.860874714258173]\n",
      "valid_roc:[0.8205985915492957]\n",
      "\n",
      "EPOCH:\t57\n",
      "train_roc:[0.858974358974359]\n",
      "valid_roc:[0.8147887323943661]\n",
      "\n",
      "EPOCH:\t58\n",
      "train_roc:[0.8605414635489823]\n",
      "valid_roc:[0.8213028169014086]\n",
      "\n",
      "EPOCH:\t59\n",
      "train_roc:[0.8624803767660911]\n",
      "valid_roc:[0.8265845070422535]\n",
      "\n",
      "EPOCH:\t60\n",
      "train_roc:[0.862821889889559]\n",
      "valid_roc:[0.8232394366197182]\n",
      "\n",
      "EPOCH:\t61\n",
      "train_roc:[0.8654438293536039]\n",
      "valid_roc:[0.8338028169014085]\n",
      "\n",
      "EPOCH:\t62\n",
      "train_roc:[0.8658679666198464]\n",
      "valid_roc:[0.835387323943662]\n",
      "\n",
      "EPOCH:\t63\n",
      "train_roc:[0.866462860447823]\n",
      "valid_roc:[0.8350352112676056]\n",
      "\n",
      "EPOCH:\t64\n",
      "train_roc:[0.8676003194800188]\n",
      "valid_roc:[0.8283450704225352]\n",
      "\n",
      "EPOCH:\t65\n",
      "train_roc:[0.8682255088270125]\n",
      "valid_roc:[0.8357394366197184]\n",
      "\n",
      "EPOCH:\t66\n",
      "train_roc:[0.8699606158252775]\n",
      "valid_roc:[0.8348591549295775]\n",
      "\n",
      "EPOCH:\t67\n",
      "train_roc:[0.8699743865157399]\n",
      "valid_roc:[0.8323943661971831]\n",
      "\n",
      "EPOCH:\t68\n",
      "train_roc:[0.8707620700101903]\n",
      "valid_roc:[0.8329225352112677]\n",
      "\n",
      "EPOCH:\t69\n",
      "train_roc:[0.8716296235093228]\n",
      "valid_roc:[0.8327464788732395]\n",
      "\n",
      "EPOCH:\t70\n",
      "train_roc:[0.8727725908177035]\n",
      "valid_roc:[0.8274647887323944]\n",
      "\n",
      "EPOCH:\t71\n",
      "train_roc:[0.874168938830593]\n",
      "valid_roc:[0.836443661971831]\n",
      "\n",
      "EPOCH:\t72\n",
      "train_roc:[0.8746151092015754]\n",
      "valid_roc:[0.8352112676056338]\n",
      "\n",
      "EPOCH:\t73\n",
      "train_roc:[0.8749951802583382]\n",
      "valid_roc:[0.8348591549295775]\n",
      "\n",
      "EPOCH:\t74\n",
      "train_roc:[0.8757938803051585]\n",
      "valid_roc:[0.8315140845070422]\n",
      "\n",
      "EPOCH:\t75\n",
      "train_roc:[0.8772811148750997]\n",
      "valid_roc:[0.8357394366197184]\n",
      "\n",
      "EPOCH:\t76\n",
      "train_roc:[0.8783497204549837]\n",
      "valid_roc:[0.8375]\n",
      "\n",
      "EPOCH:\t77\n",
      "train_roc:[0.8792558318874109]\n",
      "valid_roc:[0.835387323943662]\n",
      "\n",
      "EPOCH:\t78\n",
      "train_roc:[0.8800517777961385]\n",
      "valid_roc:[0.8385563380281691]\n",
      "\n",
      "EPOCH:\t79\n",
      "train_roc:[0.8788248092759371]\n",
      "valid_roc:[0.8382042253521127]\n",
      "\n",
      "EPOCH:\t80\n",
      "train_roc:[0.8812112699330744]\n",
      "valid_roc:[0.8392605633802817]\n",
      "\n",
      "EPOCH:\t81\n",
      "train_roc:[0.8826874879506459]\n",
      "valid_roc:[0.8403169014084507]\n",
      "\n",
      "EPOCH:\t82\n",
      "train_roc:[0.8826351593268886]\n",
      "valid_roc:[0.8410211267605634]\n",
      "\n",
      "EPOCH:\t83\n",
      "train_roc:[0.8783166707978739]\n",
      "valid_roc:[0.8406690140845069]\n",
      "\n",
      "EPOCH:\t84\n",
      "train_roc:[0.8796854774298384]\n",
      "valid_roc:[0.8399647887323943]\n",
      "\n",
      "EPOCH:\t85\n",
      "train_roc:[0.8809110688809937]\n",
      "valid_roc:[0.8334507042253522]\n",
      "\n",
      "EPOCH:\t86\n",
      "train_roc:[0.8807843785287395]\n",
      "valid_roc:[0.8339788732394366]\n",
      "\n",
      "EPOCH:\t87\n",
      "train_roc:[0.8833402186785646]\n",
      "valid_roc:[0.8404929577464788]\n",
      "\n",
      "EPOCH:\t88\n",
      "train_roc:[0.884850863422292]\n",
      "valid_roc:[0.8404929577464789]\n",
      "\n",
      "EPOCH:\t89\n",
      "train_roc:[0.8853507394860779]\n",
      "valid_roc:[0.8413732394366197]\n",
      "\n",
      "EPOCH:\t90\n",
      "train_roc:[0.8856894984714533]\n",
      "valid_roc:[0.8431338028169014]\n",
      "\n",
      "EPOCH:\t91\n",
      "train_roc:[0.8858519926189099]\n",
      "valid_roc:[0.8420774647887324]\n",
      "\n",
      "EPOCH:\t92\n",
      "train_roc:[0.8868489906083893]\n",
      "valid_roc:[0.840669014084507]\n",
      "\n",
      "EPOCH:\t93\n",
      "train_roc:[0.8883417334545156]\n",
      "valid_roc:[0.8436619718309859]\n",
      "\n",
      "EPOCH:\t94\n",
      "train_roc:[0.888782395549313]\n",
      "valid_roc:[0.8436619718309859]\n",
      "\n",
      "EPOCH:\t95\n",
      "train_roc:[0.8890881048775785]\n",
      "valid_roc:[0.8443661971830985]\n",
      "\n",
      "EPOCH:\t96\n",
      "train_roc:[0.8889779393538791]\n",
      "valid_roc:[0.8473591549295775]\n",
      "\n",
      "EPOCH:\t97\n",
      "train_roc:[0.8898647718196591]\n",
      "valid_roc:[0.847887323943662]\n",
      "\n",
      "EPOCH:\t98\n",
      "train_roc:[0.8909471480900052]\n",
      "valid_roc:[0.8496478873239437]\n",
      "\n",
      "EPOCH:\t99\n",
      "train_roc:[0.8912859070753808]\n",
      "valid_roc:[0.8475352112676056]\n",
      "\n",
      "EPOCH:\t100\n",
      "train_roc:[0.891241840865901]\n",
      "valid_roc:[0.8487676056338028]\n",
      "\n",
      "EPOCH:\t101\n",
      "train_roc:[0.8924564157646864]\n",
      "valid_roc:[0.8498239436619718]\n",
      "\n",
      "EPOCH:\t102\n",
      "train_roc:[0.8931201630449751]\n",
      "valid_roc:[0.8464788732394366]\n",
      "\n",
      "EPOCH:\t103\n",
      "train_roc:[0.8930816051116802]\n",
      "valid_roc:[0.8471830985915493]\n",
      "\n",
      "EPOCH:\t104\n",
      "train_roc:[0.8920818529841087]\n",
      "valid_roc:[0.8447183098591549]\n",
      "\n",
      "EPOCH:\t105\n",
      "train_roc:[0.8944862155388472]\n",
      "valid_roc:[0.8519366197183099]\n",
      "\n",
      "EPOCH:\t106\n",
      "train_roc:[0.8947891707290203]\n",
      "valid_roc:[0.8508802816901407]\n",
      "\n",
      "EPOCH:\t107\n",
      "train_roc:[0.8944311327769974]\n",
      "valid_roc:[0.8503521126760564]\n",
      "\n",
      "EPOCH:\t108\n",
      "train_roc:[0.8956071497424882]\n",
      "valid_roc:[0.8508802816901408]\n",
      "\n",
      "EPOCH:\t109\n",
      "train_roc:[0.8960973863229502]\n",
      "valid_roc:[0.8482394366197183]\n",
      "\n",
      "EPOCH:\t110\n",
      "train_roc:[0.8954749511140488]\n",
      "valid_roc:[0.8461267605633802]\n",
      "\n",
      "EPOCH:\t111\n",
      "train_roc:[0.8969263818887879]\n",
      "valid_roc:[0.8485915492957746]\n",
      "\n",
      "EPOCH:\t112\n",
      "train_roc:[0.8960120080420831]\n",
      "valid_roc:[0.8371478873239436]\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:\t113\n",
      "train_roc:[0.8977829188355504]\n",
      "valid_roc:[0.8434859154929578]\n",
      "\n",
      "EPOCH:\t114\n",
      "train_roc:[0.8969429067173429]\n",
      "valid_roc:[0.8466549295774648]\n",
      "\n",
      "EPOCH:\t115\n",
      "train_roc:[0.8972320912170535]\n",
      "valid_roc:[0.8450704225352113]\n",
      "\n",
      "EPOCH:\t116\n",
      "train_roc:[0.8980555785067063]\n",
      "valid_roc:[0.8459507042253521]\n",
      "\n",
      "EPOCH:\t117\n",
      "train_roc:[0.8990332975295382]\n",
      "valid_roc:[0.8503521126760564]\n",
      "\n",
      "EPOCH:\t118\n",
      "train_roc:[0.9004929907185546]\n",
      "valid_roc:[0.8538732394366197]\n",
      "\n",
      "EPOCH:\t119\n",
      "train_roc:[0.9000192789666474]\n",
      "valid_roc:[0.8519366197183099]\n",
      "\n",
      "EPOCH:\t120\n",
      "train_roc:[0.898094136440001]\n",
      "valid_roc:[0.8498239436619719]\n",
      "\n",
      "EPOCH:\t121\n",
      "train_roc:[0.9007739128039879]\n",
      "valid_roc:[0.8584507042253521]\n",
      "\n",
      "EPOCH:\t122\n",
      "train_roc:[0.9009061114324273]\n",
      "valid_roc:[0.8545774647887323]\n",
      "\n",
      "EPOCH:\t123\n",
      "train_roc:[0.9008868324657799]\n",
      "valid_roc:[0.8524647887323944]\n",
      "\n",
      "EPOCH:\t124\n",
      "train_roc:[0.901291690765375]\n",
      "valid_roc:[0.8519366197183098]\n",
      "\n",
      "EPOCH:\t125\n",
      "train_roc:[0.9023244925500563]\n",
      "valid_roc:[0.8538732394366197]\n",
      "\n",
      "EPOCH:\t126\n",
      "train_roc:[0.902316230135779]\n",
      "valid_roc:[0.8558098591549297]\n",
      "\n",
      "EPOCH:\t127\n",
      "train_roc:[0.9027926960257788]\n",
      "valid_roc:[0.8535211267605635]\n",
      "\n",
      "EPOCH:\t128\n",
      "train_roc:[0.9029166322399405]\n",
      "valid_roc:[0.8485915492957747]\n",
      "\n",
      "EPOCH:\t129\n",
      "train_roc:[0.9030460767302873]\n",
      "valid_roc:[0.8484154929577465]\n",
      "\n",
      "EPOCH:\t130\n",
      "train_roc:[0.9037125781486683]\n",
      "valid_roc:[0.8463028169014084]\n",
      "\n",
      "EPOCH:\t131\n",
      "train_roc:[0.9031975543253739]\n",
      "valid_roc:[0.8492957746478873]\n",
      "\n",
      "EPOCH:\t132\n",
      "train_roc:[0.9049602027045637]\n",
      "valid_roc:[0.8531690140845071]\n",
      "\n",
      "EPOCH:\t133\n",
      "train_roc:[0.904029304029304]\n",
      "valid_roc:[0.8466549295774648]\n",
      "\n",
      "EPOCH:\t134\n",
      "train_roc:[0.9024291497975709]\n",
      "valid_roc:[0.8389084507042254]\n",
      "\n",
      "EPOCH:\t135\n",
      "train_roc:[0.9049409237379163]\n",
      "valid_roc:[0.8457746478873239]\n",
      "\n",
      "EPOCH:\t136\n",
      "train_roc:[0.9042992095623674]\n",
      "valid_roc:[0.8441901408450704]\n",
      "\n",
      "EPOCH:\t137\n",
      "train_roc:[0.906260155884216]\n",
      "valid_roc:[0.848943661971831]\n",
      "\n",
      "EPOCH:\t138\n",
      "train_roc:[0.9068660662645626]\n",
      "valid_roc:[0.8501760563380282]\n",
      "\n",
      "EPOCH:\t139\n",
      "train_roc:[0.9051419758186676]\n",
      "valid_roc:[0.8471830985915493]\n",
      "\n",
      "EPOCH:\t140\n",
      "train_roc:[0.9031617505301714]\n",
      "valid_roc:[0.8470070422535212]\n",
      "\n",
      "EPOCH:\t141\n",
      "train_roc:[0.9041670109339283]\n",
      "valid_roc:[0.8519366197183099]\n",
      "\n",
      "EPOCH:\t142\n",
      "train_roc:[0.9071056762786085]\n",
      "valid_roc:[0.852112676056338]\n",
      "\n",
      "EPOCH:\t143\n",
      "train_roc:[0.9077942108017296]\n",
      "valid_roc:[0.8528169014084506]\n",
      "\n",
      "EPOCH:\t144\n",
      "train_roc:[0.9082238563441571]\n",
      "valid_roc:[0.8533450704225353]\n",
      "\n",
      "EPOCH:\t145\n",
      "train_roc:[0.906100415874852]\n",
      "valid_roc:[0.8403169014084507]\n",
      "\n",
      "EPOCH:\t146\n",
      "train_roc:[0.9080448373681456]\n",
      "valid_roc:[0.847887323943662]\n",
      "\n",
      "EPOCH:\t147\n",
      "train_roc:[0.9072516455975104]\n",
      "valid_roc:[0.8538732394366197]\n",
      "\n",
      "EPOCH:\t148\n",
      "train_roc:[0.9086397311961222]\n",
      "valid_roc:[0.8473591549295775]\n",
      "\n",
      "EPOCH:\t149\n",
      "train_roc:[0.9070643642072214]\n",
      "valid_roc:[0.8556338028169014]\n",
      "\n",
      "EPOCH:\t150\n",
      "train_roc:[0.9097468947093008]\n",
      "valid_roc:[0.854225352112676]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_param ={}\n",
    "best_param[\"roc_epoch\"] = 0\n",
    "best_param[\"loss_epoch\"] = 0\n",
    "best_param[\"valid_roc\"] = 0\n",
    "best_param[\"valid_loss\"] = 9e8\n",
    "\n",
    "for epoch in range(epochs):    \n",
    "    train_roc, train_loss = eval(model, train_df)\n",
    "    valid_roc, valid_loss = eval(model, valid_df)\n",
    "    train_roc_mean = np.array(train_roc).mean()\n",
    "    valid_roc_mean = np.array(valid_roc).mean()\n",
    "    \n",
    "#     tensorboard.add_scalars('ROC',{'train_roc':train_roc_mean,'valid_roc':valid_roc_mean},epoch)\n",
    "#     tensorboard.add_scalars('Losses',{'train_losses':train_loss,'valid_losses':valid_loss},epoch)\n",
    "\n",
    "    if valid_roc_mean > best_param[\"valid_roc\"]:\n",
    "        best_param[\"roc_epoch\"] = epoch\n",
    "        best_param[\"valid_roc\"] = valid_roc_mean\n",
    "        if valid_roc_mean > 0.85:\n",
    "             torch.save(model, 'saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(epoch)+'.pt')             \n",
    "    if valid_loss < best_param[\"valid_loss\"]:\n",
    "        best_param[\"loss_epoch\"] = epoch\n",
    "        best_param[\"valid_loss\"] = valid_loss\n",
    "\n",
    "    print(\"EPOCH:\\t\"+str(epoch)+'\\n'\\\n",
    "        +\"train_roc\"+\":\"+str(train_roc)+'\\n'\\\n",
    "        +\"valid_roc\"+\":\"+str(valid_roc)+'\\n'\\\n",
    "#         +\"train_roc_mean\"+\":\"+str(train_roc_mean)+'\\n'\\\n",
    "#         +\"valid_roc_mean\"+\":\"+str(valid_roc_mean)+'\\n'\\\n",
    "        )\n",
    "    if (epoch - best_param[\"roc_epoch\"] >18) and (epoch - best_param[\"loss_epoch\"] >28):        \n",
    "        break\n",
    "        \n",
    "    torch.manual_seed(epoch)    \n",
    "    train(model, train_df, optimizer, loss_function)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch:121\n",
      "test_roc:[0.8561717352415027]\n",
      "test_roc_mean: 0.8561717352415027\n"
     ]
    }
   ],
   "source": [
    "# evaluate model\n",
    "best_model = torch.load('saved_models/model_'+prefix_filename+'_'+start_time+'_'+str(best_param[\"roc_epoch\"])+'.pt')     \n",
    "\n",
    "best_model_dict = best_model.state_dict()\n",
    "best_model_wts = copy.deepcopy(best_model_dict)\n",
    "\n",
    "model.load_state_dict(best_model_wts)\n",
    "(best_model.align[0].weight == model.align[0].weight).all()\n",
    "test_roc, test_losses = eval(model, test_df)\n",
    "\n",
    "print(\"best epoch:\"+str(best_param[\"roc_epoch\"])\n",
    "      +\"\\n\"+\"test_roc:\"+str(test_roc)\n",
    "      +\"\\n\"+\"test_roc_mean:\",str(np.array(test_roc).mean())\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
